Explore data

eda
Author

Renato Hermoza

Published

September 30, 2022

Get data

Last execution time: 12/05/2025 05:44:22
Products type filter
explore_types = ['frutas', 'lacteos', 'verduras', 'embutidos', 'panaderia', 'desayuno', 'congelados', 'abarrotes',
                 'aves', 'carnes', 'pescados']
Data table
path = Path('../../output')
csv_files = L(path.glob('*.csv')).filter(lambda o: os.stat(o).st_size>0)
pat_store = re.compile('(.+)\_\d+')
pat_date = re.compile('.+\_(\d+)')
df = (
    pd.concat([pd.read_csv(o).assign(store=pat_store.match(o.stem)[1], date=pat_date.match(o.stem)[1])
               for o in csv_files], ignore_index=True)
    .pipe(lambda d: d.assign(
        name=d.name.str.lower()+' ('+d.store+')',
        sku=d.id.where(d.sku.isna(), d.sku).astype(int),
        date=pd.to_datetime(d.date)
    ))
    .drop('id', axis=1)
    .loc[lambda d: d.category.str.contains('|'.join(explore_types))]
    # Filter products with recent data
#     .loc[lambda d: d.name.isin(d.groupby('name').date.max().loc[ge(datetime.now()-timedelta(days=30))].index)]
    # Filter empty prices
    .loc[lambda d: d.price>0]
)
print(df.shape)
df.sample(3)
(1384547, 8)
sku name brand category uri price store date
1994800 10705780 trozos de atún en aceite vegetal campomar lata... CAMPOMAR https://www.plazavea.com.pe/abarrotes https://www.plazavea.com.pe/trozos-de-atun-en-... 5.59 plaza_vea 2023-05-25
2524502 13437 osobuco de res (plaza_vea) EL BUEN CORTE https://www.plazavea.com.pe/carnes-aves-y-pesc... https://www.plazavea.com.pe/osobuco-de-res-el-... 22.40 plaza_vea 2022-10-10
1857057 11124692 leche uht laive light sin lactosa bolsa 900ml ... LAIVE https://www.plazavea.com.pe/lacteos-y-huevos https://www.plazavea.com.pe/leche-uht-laive-li... 5.20 plaza_vea 2024-02-12

Top changes (ratio)

Code
top_changes = (df
 # Use last 30 days of data to compare prices
 .loc[lambda d: d.date>=(datetime.now()-timedelta(days=30))]
 .sort_values('date')
 # Get percentage change
 .assign(change=lambda d: d
     .groupby(['store','sku'], as_index=False)
     .price.transform(lambda d: (d-d.shift())/d.shift())
 )
 .groupby(['store','sku'], as_index=False)
 .agg({'price':'last', 'change':'mean', 'date':'last'})
 .rename({'price':'last_price', 'date':'last_date'}, axis=1)
 .dropna()
 .loc[lambda d: d.last_date==d.last_date.max()]
 .loc[lambda d: d.change.abs().sort_values(ascending=False).index]
)
top_changes.head(3)
store sku last_price change last_date
4917 plaza_vea 11339752 19.0 0.293729 2025-05-12
1698 plaza_vea 24247 18.9 0.227273 2025-05-12
6084 plaza_vea 11806440 16.9 0.210084 2025-05-12
Code
def plot_changes(df_changes, title):
    selection = alt.selection_point(fields=['name'], bind='legend')
    dff = df_changes.drop('change', axis=1).merge(df, on=['store','sku'])
    return (dff
     .pipe(alt.Chart)
     .mark_line(point=True)
     .encode(
         x='date',
         y='price',
         color=alt.Color('name').scale(domain=sorted(dff.name.unique().tolist())),
         tooltip=['name','price','last_price']
     )
     .add_params(selection)
     .transform_filter(selection)
     .interactive()
     .properties(width=650, title=title)
     .configure_legend(orient='top', columns=3)
    )
Code
top_changes.head(10).pipe(plot_changes, 'Top changes')
Code
(top_changes
 .sort_values('change')
 .head(10)
 .pipe(plot_changes, 'Top drops')
)
Code
(top_changes
 .sort_values('change')
 .tail(10)
 .pipe(plot_changes, 'Top increases')
)

Top changes (absolute values)

Code
top_changes_abs = (df
 # Use last 30 days of data to compare prices
 .loc[lambda d: d.date>=(datetime.now()-timedelta(days=30))]
 .sort_values('date')
 # Get percentage change
 .assign(change=lambda d: d
     .groupby(['store','sku'], as_index=False)
     .price.transform(lambda d: (d-d.shift()).iloc[-1])
 )
 .groupby(['store','sku'], as_index=False)
 .agg({'price':'last', 'change':'mean', 'date':'last'})
 .rename({'price':'last_price', 'date':'last_date'}, axis=1)
 .dropna()
 .loc[lambda d: d.last_date==d.last_date.max()]
 .loc[lambda d: d.change.abs().sort_values(ascending=False).index]
)
top_changes_abs.head(3)
store sku last_price change last_date
2250 plaza_vea 62756 51.8 16.3 2025-05-12
1288 plaza_vea 10742 49.9 -13.0 2025-05-12
226 plaza_vea 1161 41.2 12.7 2025-05-12
Code
top_changes_abs.head(10).pipe(plot_changes, 'Top changes')
Code
(top_changes_abs
 .sort_values('change')
 .head(10)
 .pipe(plot_changes, 'Top drops')
)
Code
(top_changes_abs
 .sort_values('change')
 .tail(10)
 .pipe(plot_changes, 'Top increases')
)

Search specific products

Code
(df
 .loc[df.name.isin(names)]
 .pipe(alt.Chart)
 .mark_line(point=True)
 .encode(x='date', y='price', color='name', tooltip=['name','price'])
 .properties(width=650, title='Pollo')
 .interactive()
 .configure_legend(orient='top', columns=3)
)
Code
(df
 .loc[df.name.isin(names)]
 .pipe(alt.Chart)
 .mark_line(point=True)
 .encode(x='date', y='price', color='name', tooltip=['name','price'])
 .properties(width=650, title='Palta')
 .interactive()
 .configure_legend(orient='top', columns=3)
)
Code
(df
 .loc[df.name.isin(names)]
 .pipe(alt.Chart)
 .mark_line(point=True)
 .encode(x='date', y='price', color='name', tooltip=['name','price'])
 .properties(width=650, title='Aceite')
 .interactive()
 .configure_legend(orient='top', columns=3)
)
Code
(df
 .loc[df.name.isin(names)]
 .pipe(alt.Chart)
 .mark_line(point=True)
 .encode(x='date', y='price', color='name', tooltip=['name','price'])
 .properties(width=650, title='Aceite')
 .interactive()
 .configure_legend(orient='top', columns=3)
)
Code
(df
 .loc[df.name.isin(names)]
 .pipe(alt.Chart)
 .mark_line(point=True)
 .encode(x='date', y='price', color='name', tooltip=['name','price'])
 .properties(width=650, title='Aceite')
 .interactive()
 .configure_legend(orient='top', columns=3)
)